import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;

import org.omg.CORBA.PUBLIC_MEMBER;

public class CopyOfRwr_GM extends TopicModel {

	enum Model{group,user,item_pre,item_mod};//不同的模型
	enum Policy{lm,avg,wtd};//不同的策略
	
	public float[][] preGMRating;
	public float[][] calGMRating;
	public float[] calGMAvgRating;
	public float[] calUMAvgRating;
	public float[][] preUMRating;
	public int[] nZ;

	public float[][] sGM;
	public float[][] sMU;
	public float[][] sUG;
	public float[][] sMUG;
	public float[][] sMGU;
	public float[][] sMG;
    public float[][] sGU;
	public float[][] sUM;


	public int M;
	public int W;
	public int Z;
	public int G;
	public int U;
	public int GT;
	public int MT;
	public int S;
	public float divisor=20;
	public int I=400;//隨機矩陣迭代次數

	public float beta;
	public float alpha;

	HashMap<Integer, Movie> movieListNew = new HashMap<Integer, Movie>();//mid->movie对象 (train)
	HashMap<Integer, Movie> movieListCheck = new HashMap<Integer, Movie>();//mid->movie对象 （check）
	HashMap<Integer, Group> groupListNew = new HashMap<Integer, Group>();//gid->group对象 （train）
	HashMap<Integer, Group> groupListCheck = new HashMap<Integer, Group>();//gid->group对象 (check)
	List<Integer> userListNew = new ArrayList<Integer>();//train中所有uid的列表
	List<Integer> userListCheck = new ArrayList<Integer>();//check中所有uid的列表

	Movie[] allMovies;
	Group[] allGroups;
	Integer[] allUsers;
	public HashMap<Integer, HashMap<Integer, Integer>> zwc = new HashMap<Integer, HashMap<Integer, Integer>>();//topic->wid->count
	public HashMap<Integer, HashMap<Integer, Float>> pUZ = new HashMap<Integer, HashMap<Integer, Float>>(); // uid->topic->p
	public HashMap<Integer, HashMap<Integer, Float>> pGZ = new HashMap<Integer, HashMap<Integer, Float>>();// gid->topic->p

	public HashMap<Integer, HashMap<Integer, Float>> test_gmRating = new HashMap<Integer, HashMap<Integer, Float>>();// train+check中的gid->mid->rating
	 
	float max=Float.MIN_VALUE,min=Float.MAX_VALUE;
	Random f = new Random();
	float[][] ss;
	int[][] ssIndex;
	float[] temp;

	public CopyOfRwr_GM(int z, double a, double b) {
		alpha = (float)a;
		beta = (float)b;
		Z = z;
	}

	@Override
	public void initialize() {
		System.out.println("Initializing...");
		Random r = new Random();

		nZ = new int[Z];
		sGM = new float[G][M];
		sMG = new float[M][G];
		sMU = new float[M][U];
		sUM = new float[U][M];
		sUG = new float[U][G];
		sGU = new float[G][U];
		sMUG = new float[M][G];
		sMGU = new float[M][U];
		preUMRating = new float[U][M];
		preGMRating = new float[G][M];// 1.13修改，把GT、MT分别改为G、M
		calGMRating = new float[G][M];// 1.13修改，由MT改为M
		calGMAvgRating = new float[G];// 3.6修改
		calUMAvgRating = new float[U];

		for (int z = 0; z < Z; z++) {
			HashMap<Integer, Integer> wc = new HashMap<Integer, Integer>();// wid->count
			zwc.put(z, wc);
		}

		for (Movie movie : allMovies) {
			for (Map.Entry<Integer, Integer> word : movie.words.entrySet()) {
				int z = r.nextInt(this.Z);
				if (!movie.countToTopic.containsKey(z))
					movie.countToTopic.put(z, 0);
				if(!zwc.get(z).containsKey(word.getKey()))
					zwc.get(z).put(word.getKey(), 0);
				word.setValue(z);
				// update counts
				zwc.get(z).put(word.getKey(), zwc.get(z).get(word.getKey())+1);
				nZ[z]++;
				movie.countToTopic.put(z, movie.countToTopic.get(z) + 1);
				movie.countToAllTopic++;
			}

			for (int uid : movie.users.keySet()) {
				HashMap<Integer, Float> zp = new HashMap<Integer, Float>();
				for (int z = 0; z < Z; z++)
					zp.put(z, 0.0f);
				pUZ.put(uid, zp);
			}
		}
		for (Group group : allGroups) {
			HashMap<Integer, Float> zp = new HashMap<Integer, Float>();
			for (int z = 0; z < Z; z++)
				zp.put(z, 0.0f);
			pGZ.put(group.gid, zp);
		}
	}
	
	@Override
	public void doSampling() {
		for (Movie movie : allMovies) {
			for (Entry<Integer, Integer> word : movie.words.entrySet()) {
				sample(movie, word);
			}
		}
		likelihood();
	}

	int nZW(int z, int w) {
		if (!zwc.get(z).containsKey(w))
			zwc.get(z).put(w, 0);
		return zwc.get(z).get(w);
	}

	public void sample(Movie movie, Entry<Integer, Integer> word) {
		final int w = word.getKey();
		int z = word.getValue();

		if (!zwc.get(z).containsKey(w))
			zwc.get(z).put(w, 0);

		// decrement counts
		int c = zwc.get(z).get(w);// 原count
		zwc.get(z).put(w, c - 1);
		nZ[z]--;
		movie.countToTopic.put(z, movie.countToTopic.get(z) - 1);
		movie.countToAllTopic--;
		// sample new value for topic
		float pTotal = 0.0f;
		float[] p = new float[Z];

		for (z = 0; z < Z; z++) {
			p[z] = (nZW(z, w) + beta) / (nZ[z] + W * beta)
					* (movie.getTopicCount(z) + alpha);

			pTotal += p[z];
		}
		Random r = new Random();
		float e = r.nextFloat() * pTotal;

		float f = 0;
		for (z = 0; z < Z; z++) {
			f += p[z];

			if (f > e) {
				break;
			}
		}
		if (z == Z)
			z = word.getValue();
		// set new assignments
		word.setValue(z);

		// increment counts
		if (!zwc.get(z).containsKey(w))
			zwc.get(z).put(w, 0);
		c = zwc.get(z).get(w);// 原count
		zwc.get(z).put(w, c + 1);
		nZ[z]++;
		if (!movie.countToTopic.containsKey(z))
			movie.countToTopic.put(z, 0);
		movie.countToTopic.put(z, movie.countToTopic.get(z) + 1);
		movie.countToAllTopic++;

	}
	

	/**
	 * 
	 * @param pre
	 * @param cur
	 * @return
	 */
	public boolean isConverged(float[] pre, float[] cur) {
		float max = 0;
		for (int i = 0; i < pre.length; i++) {
			max = Math.max(max, Math.abs(pre[i] - cur[i]) / pre[i]);
		}
		return max <= 0.01;// 判断random walk 收敛的条件
	}

	/**
	 * obtaining sUM,sMU by pMZ and pUZ
	 * 三种策略可共用这些矩阵，因此只需计算一遍
	 */
	public void calculatePublicModel(){

		// distributions of topics over movies
		float p;
		for (Movie movie : allMovies) {
			for (int z = 0; z < Z; z++) {
				p = (movie.getTopicCount(z) + alpha)
						/ (movie.countToAllTopic + Z * alpha);
				movie.pmz.put(z, p);
				if (p == 0) {
					System.out.println(p + "+++++++++++++");
				}
			}
		}

		// distributions of topics over users by count 
		List<Movie> movies;
		float[] sumRatings;
		float[] cu = new float[U];
		for (Entry<Integer, HashMap<Integer, Float>> user : pUZ.entrySet()) {
			int uid = user.getKey();
			movies = findAllMoviesByUser(uid);
			sumRatings = new float[movies.size()];			
			for (int i = 0; i < movies.size(); i++) {
				sumRatings[i] = movies.get(i).getSumRating(); 
			}
			for(Movie movie:movies){
				cu[userListNew.indexOf(user.getKey())] += movie.users.get(uid)/(sumRatings[movies.indexOf(movie)]*movie.countToAllTopic);
			}

			for (int z = 0; z < Z; z++) {
				float cuz = 0;
				for (Movie movie : movies) {
					if(!movie.countToTopic.containsKey(z))
						continue;
					cuz += movie.users.get(uid) * movie.countToTopic.get(z) / sumRatings[movies.indexOf(movie)];					
				}
				float puz =(cuz+alpha)/(cu[userListNew.indexOf(user.getKey())]+Z*alpha);
				user.getValue().put(z, puz);
				if (Float.isNaN(puz)) {
					// System.out.println(group.gid+","+z);
					System.out.println("NaN:puz-"+z);
					System.exit(-1);
				}
				
			}
		}		
		System.out.println("["+(new Date())+"]_"+"distributions of topics over users is completed");
		// sUM,sMU
		for (int u = 0; u < U; u++) {
			for (int m = 0; m < M; m++) {
				sMU[m][u] = 0;
				float put = 0;
				float pmt = 0;
				for (int z = 0; z < Z; z++) {
					sMU[m][u] = sMU[m][u] + pUZ.get(allUsers[u]).get(z)
							* allMovies[m].pmz.get(z);
					put = put + (float)Math.pow(pUZ.get(allUsers[u]).get(z), 2);
					pmt = pmt + (float)Math.pow(allMovies[m].pmz.get(z), 2);
				}
				put = (float)Math.sqrt(put);
				pmt = (float)Math.sqrt(pmt);
				sMU[m][u] = sMU[m][u] / (put * pmt);
				sUM[u][m] = sMU[m][u];//used by 2nd model
			}
		}
		System.out.println("["+(new Date())+"]_"+"sUM and sMU completed");
		// normalize sUM
		for (int j = 0; j < M; j++) {
			float sumsUM = 0;
			for (int i = 0; i < U; i++) {
				sumsUM = sumsUM + sUM[i][j];

			}
			for (int i = 0; i < U; i++) {
				sUM[i][j] = sUM[i][j] / sumsUM;
			}
		}
		System.out.println("["+(new Date())+"]_"+"normalized sUM completed");
		// normalize sMU
		for (int j = 0; j < U; j++) {
			float sumsMU = 0;
			for (int i = 0; i < M; i++) {
				sumsMU = sumsMU + sMU[i][j];

			}
			for (int i = 0; i < M; i++) {
				sMU[i][j] = sMU[i][j] / sumsMU;
			}
		}
		System.out.println("["+(new Date())+"]_"+"normalized sMU completed");
	}
	
	
	/**
	 * calculate test_gmRating and max,min
	 * @param policy
	 */
	public void calculateTestGMRating(Policy policy){
		switch(policy){
		case wtd:
			for (Group testGroup : groupListCheck.values()) {
				float sumOfNum = 0;
				for (int n : groupListNew.get(testGroup.gid).users.values()) 
					sumOfNum += n;
				for(int n:testGroup.users.values())
					sumOfNum += n;
				for (Movie testMovie : movieListCheck.values()) {
					float tgmr = -1;
					for(int uid:groupListNew.get(testGroup.gid).users.keySet()){
						if(!movieListNew.get(testMovie.mid).users.containsKey(uid))
							continue;
						float r = movieListNew.get(testMovie.mid).users.get(uid);
						float total = groupListNew.get(testGroup.gid).users.get(uid);
						if(testGroup.users.containsKey(uid))
							total += testGroup.users.get(uid);
						tgmr += total / sumOfNum * r;
					}
					for (int uid : testGroup.users.keySet()) {
						if(!testMovie.users.containsKey(uid))
							continue;
						float r = testMovie.users.get(uid);
						float total = testGroup.users.get(uid);
						if(groupListNew.get(testGroup.gid).users.containsKey(uid))
							total += groupListNew.get(testGroup.gid).users.get(uid);
						tgmr += total / sumOfNum * r;
					}
					if (tgmr == -1)
						continue;
					tgmr++;
					if (!test_gmRating.containsKey(testGroup.gid))
						test_gmRating.put(testGroup.gid,
								new HashMap<Integer, Float>());
					test_gmRating.get(testGroup.gid).put(testMovie.mid, tgmr);
					max = Math.max(max, tgmr);
					min = Math.min(min, tgmr);
				}
			}
			break;
		case lm:
			for(Group testGroup:groupListCheck.values()){
				for (Movie testMovie : movieListCheck.values()) {
					float tgmr = Integer.MAX_VALUE;
					for(int uid:groupListNew.get(testGroup.gid).users.keySet()){
						if(!movieListNew.get(testMovie.mid).users.containsKey(uid))
							continue;
						float r = movieListNew.get(testMovie.mid).users.get(uid);
						tgmr = Math.min(tgmr, r);
					}
					for (int uid : testGroup.users.keySet()) {
						if(!testMovie.users.containsKey(uid))
							continue;
						float r = testMovie.users.get(uid);
						tgmr += Math.min(tgmr, r);
					}
					if(tgmr == Integer.MAX_VALUE)
						continue;
					if (!test_gmRating.containsKey(testGroup.gid))
						test_gmRating.put(testGroup.gid,
								new HashMap<Integer, Float>());
					test_gmRating.get(testGroup.gid).put(testMovie.mid, tgmr);
					//System.out.print(tgmr+",");
					max = Math.max(max, tgmr);
					min = Math.min(min, tgmr);
				}
				//System.out.println();
			}
			break;
		case avg:
			for(Group testGroup:groupListCheck.values()){
				for (Movie testMovie : movieListCheck.values()) {
					float tgmr = -1;
					for(int uid:groupListNew.get(testGroup.gid).users.keySet()){
						if(!movieListNew.get(testMovie.mid).users.containsKey(uid))
							continue;
						tgmr += movieListNew.get(testMovie.mid).users.get(uid);
					}
					for (int uid : testGroup.users.keySet()) {
						if(!testMovie.users.containsKey(uid))
							continue;
						tgmr += testMovie.users.get(uid);
					}
					if(tgmr == -1)
						continue;
					tgmr++;
					tgmr /= (groupListNew.get(testGroup.gid).users.size()+testGroup.users.size());
					if (!test_gmRating.containsKey(testGroup.gid))
						test_gmRating.put(testGroup.gid,
								new HashMap<Integer, Float>());
					test_gmRating.get(testGroup.gid).put(testMovie.mid, tgmr);
					//System.out.print(tgmr+",");
					max = Math.max(max, tgmr);
					min = Math.min(min, tgmr);
				}
				//System.out.println();
			}
			break;
		}
		System.out.println("["+(new Date())+"]_"+"test_gmrating completed!");
	}
	
	/**
	 * 根据不同的policy整合模型
	 * @param policy
	 * @throws IOException
	 */
	public void rATM_unRWR(Policy policy) throws IOException {

		// distributions of topics over groups
		for (Group group : allGroups) {
			float sum = 0;
			for (int n : group.users.values()) {
				sum += n;
			}
			float pgz = 0;
			for (int z = 0; z < Z; z++) {
				switch(policy){
				case wtd:
					for (Entry<Integer, Integer> user : group.users.entrySet()) {
						if (user.getValue() == 0)
							continue;
						pgz += user.getValue() / sum * pUZ.get(user.getKey()).get(z);//weighted
					}
					break;
				case lm:
					List<Integer> users = new ArrayList<Integer>();
					int min = M;
					for (Entry<Integer, Integer> user : group.users.entrySet()) {
						if (user.getValue() == 0)
							continue;
						else if(user.getValue() == min)
							users.add(user.getKey());
						else if(user.getValue()<min){
							min = user.getValue();
							users.clear();
							users.add(user.getKey());
						}
					}
					for(int uid:users)
						pgz += pUZ.get(uid).get(z);
					pgz /= users.size();
					break;
				case avg:
					for(Entry<Integer, Integer> user : group.users.entrySet())
						pgz += pUZ.get(user.getKey()).get(z);
					pgz /= group.users.size();
					break;
				}
				pGZ.get(group.gid).put(z, pgz);
				if (Float.isNaN(pgz)) {
					// System.out.println(group.gid+","+z);
					System.out.println("NaN:pgz-"+z);
					System.exit(-1);
				}
			}
		}
		System.out.println("["+(new Date())+"]_"+"distributions of topics over groups is completed");
		
		// similarity between groups and movies
		for (int g = 0; g < G; g++) {
			for (int m = 0; m < M; m++) {
				sGM[g][m] = 0;
				float pgt = 0;
				float pmt = 0;
				for (int z = 0; z < Z; z++) {
					sGM[g][m] = sGM[g][m] + pGZ.get(allGroups[g].gid).get(z)
							* allMovies[m].pmz.get(z);
					pmt = pmt + allMovies[m].pmz.get(z)
							* allMovies[m].pmz.get(z);
					pgt = pgt + pGZ.get(allGroups[g].gid).get(z)
							* pGZ.get(allGroups[g].gid).get(z);
				}
				pmt = (float)Math.sqrt(pmt);
				pgt = (float)Math.sqrt(pgt);

				sGM[g][m] = sGM[g][m] / (pmt * pgt);
				sMG[m][g] = sGM[g][m];//used by 2nd model
				if (Float.isNaN(sGM[g][m])) {
					// System.out.println(group.gid+","+z);
					System.out.println("NaN:sGM-"+g+","+m);
					System.exit(-1);
				}
			}
		}
		System.out.println("["+(new Date())+"]_"+"sGU and sUG are completed");
		
		// normalize sGM
		
		for (int j = 0; j < M; j++) {
			float sumsGM = 0;
			for (int i = 0; i < G; i++) {
				sumsGM = sumsGM + sGM[i][j];

			}
			for (int i = 0; i < G; i++) {
				sGM[i][j] = sGM[i][j] / sumsGM;
			}
		}
		System.out.println ("["+(new Date())+"]_"+"normalized sGM completed");
      // normalize sMG
		for (int j = 0; j < G; j++) {
			float sumsMG = 0;
			for (int i = 0; i < M; i++) {
				sumsMG= sumsMG + sMG[i][j];

			}
			for (int i = 0; i < M; i++) {
				sMG[i][j] = sMG[i][j] / sumsMG;
			}
		}
		System.out.println ("["+(new Date())+"]_"+"normalized sMG completed");

		// group-based model(1st model)
		firstModel(policy);
	
		//user-based model(2nd model)
//		secondModel(policy);

		// item-based prediction and item-based model(3rd + 4th model)
//		thirdAndFourthModel(policy);

		
	}
	
	/**
	 * print all resulting matrix and the specified matrix m
	 * @param m
	 */
	private void printResults(Model model,Policy policy){
		String basePath = "/home/qsy/fss/UtmRwr_Model/"+model+"/"+policy;
		try {
			matrixToFile(sMU, basePath+"/sMU_result.csv");
			matrixToFile(sUG, basePath+"/sUG_result.csv");
			matrixToFile(sGM, basePath+"/sGM_result.csv");
			matrixToFile(sMUG, basePath+"/sMUG_result.csv");
			
			matrixToFile(sMG, basePath+"/sMG_result.csv");
			matrixToFile(sGU, basePath+"/sGU_result.csv");
			matrixToFile(sUM, basePath+"/sUM_result.csv");
			matrixToFile(sMGU, basePath+"/sMGU_result.csv");
			
			matrixToFile(ss, basePath+"/ss_result.csv");
			matrixToFile(preGMRating, basePath+"/preGMRating_result.csv");
			matrixToFile(calGMRating, basePath+"/calGMRating_result.csv");
			matrixToFile(preUMRating, basePath+"/preUMRating_result.csv");
			
//			matrixToFile(m, basePath+"/sUU_result.csv");
			
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	/**
	 * calculate rating of all movies rated by all groups<br>
	 * calGMRating is only used in 1st and 4th model
	 * @param policy
	 */
	public void calGMRating(Policy policy){
		// no rating = -1;

		for (int g = 0; g < G; g++) {
			float sumOfNum = 0;
			for (int n : allGroups[g].users.values()) {
				sumOfNum += n;
			}
			for (int m = 0; m < M; m++) {
				float cGMRating = (policy == Policy.lm) ? Integer.MAX_VALUE : -1;
				for (Entry<Integer, Integer> user : allGroups[g].users.entrySet()) {
					if (!allMovies[m].users.containsKey(user.getKey()))
						continue;
					switch (policy) {
					case wtd:
						cGMRating += user.getValue() / sumOfNum
								* allMovies[m].users.get(user.getKey());
						break;
					case lm:
						cGMRating = Math.min(cGMRating,
								allMovies[m].users.get(user.getKey()));
						break;
					case avg:
						cGMRating += allMovies[m].users.get(user.getKey());
						break;
					}
				}
				if (cGMRating == Integer.MAX_VALUE)
					cGMRating = -1;
				if (cGMRating != -1) {
					if (policy != CopyOfRwr_GM.Policy.lm)
						cGMRating++;
					if (policy == CopyOfRwr_GM.Policy.avg)
						cGMRating /= allGroups[g].users.size();
				}

				calGMRating[g][m] = cGMRating;
				for (int i = 0; i < calGMRating[g].length; i++) {
					if (calGMRating[g][i] != -1)
						calGMAvgRating[g] += calGMRating[g][i];
				}
				calGMAvgRating[g] /= sumOfNum;
			}
		}
	}

	/**
	 * group-based model(1st)
	 * @param policy
	 */
	private void firstModel(Policy policy) {
		// TODO Auto-generated method stub
		S = G;
		calGMRating(policy);
		float[] sgm = new float[G + M]; // random matrix

		// 初始化随机矩阵
		ss = new float[S][G];
		ssIndex = new int[S][G];
		for (int g = 0; g < G; g++) {
			System.out.print(","+g);
			for (int Urwr = 0; Urwr < (G + M); Urwr++)
				sgm[Urwr] = f.nextFloat(); // random matrix
			for (int iter = 0; iter < 200; iter++) {
				temp = new float[G + M];
				//temp[0...G-1] = sGM x sgm[G...G+M-1]
				for (int i = 0; i < G; i++) {
					for (int j = 0; j < M; j++) {
						temp[i] += sGM[i][j] * sgm[G+j];
					}
					temp[i] = 0.95f * temp[i] + 0.05f * (g == i ? 1 : 0);
				}
				//temp[G...G+M-1] = sMUG x sgm[0...G-1]
				for(int i=0;i<M;i++){
					for(int j=0;j<G;j++){
						temp[G+i] += sMG[i][j] * sgm[j];
					}
					temp[G+i] *= 0.95f;
				}
				if(isConverged(sgm, temp)){
					sgm = temp;
					System.out.print("c");
					break;
				}
				else{
					sgm = temp;
				}
			}
			for (int i = 0; i < S; i++) {
				int index = findIndexOfLargest(sgm, 0, G);
				ss[i][g] = sgm[index];
				ssIndex[i][g] = index;
				sgm[index] = 0;
			}
		}
		System.out.println();
		System.out.println("["+(new Date())+"]_"+"ss for group is completed");

		// predicting rating of groups
		for (int g1 = 0; g1 < G; g1++) {
			float fz = 0, fm = 0;
			for (int m = 0; m < M; m++) {
				for (int i = 0; i < S; i++) {
					if (calGMRating[ssIndex[i][g1]][m] == -1)
						continue;
					fz += ss[i][g1] * (calGMRating[ssIndex[i][g1]][m] - calGMAvgRating[ssIndex[i][g1]]);
					fm += Math.abs(ss[i][g1]);
				}
				preGMRating[g1][m] = calGMAvgRating[g1] + fz / fm;
			}
		}

		printResults(Model.group,policy);
		System.out.println("group-based model");
		maeAucPcsMapRecall();
	}
	
	/**
	 * user-based model(2nd)
	 * @param policy
	 */
	private void secondModel(Policy policy) {
		// TODO Auto-generated method stub
		// random walk with restart aggregate prediction

		float[] sum = new float[U + M]; // random matrix

		S = U;
		ss = new float[S][U];
		ssIndex = new int[S][U];

		for (int u = 0; u < U; u++) {
			System.out.print(","+u);
			for (int Urwr = 0; Urwr < (U + M); Urwr++)
				sum[Urwr] = f.nextFloat(); // random matrix
			for (int iter = 0; iter < I; iter++) {
				temp = new float[U + M];
				//temp[0...U-1] = sUM x sum[U...U+M-1]
				for (int i = 0; i < U; i++) {
					for (int j = 0; j < M; j++) {
						temp[i] += sUM[i][j] * sum[U+j];
					}
					temp[i] = 0.95f * temp[i] + 0.05f * (u == i ? 1 : 0);
				}
				//temp[U...U+M-1] = sMGU x sum[0...U-1]
				for(int i=0;i<M;i++){
					for(int j=0;j<U;j++){
						temp[U+i] += sMU[i][j] * sum[j];
					}
					temp[U+i] *= 0.95f;
				}
				if(isConverged(sum, temp)){
					sum = temp;
					System.out.print("c");
					break;
				}
				else{
					sum = temp;
				}
			}
			for (int i = 0; i < S; i++) {
				int indexPre = findIndexOfLargest(sum, 0, U);
				ss[i][u] = sum[indexPre];
				ssIndex[i][u] = indexPre;
				sum[indexPre]=0;
			}
		}
		System.out.println();
		System.out.println("["+(new Date())+"]_"+"ss for user is completed");

		// calculating average ratings of movies by users
		for (int uid : userListNew) {
			float s = 0, num = 0;
			for (Movie movie : allMovies) {
				if (movie.users.containsKey(uid)) {
					s += movie.users.get(uid);
					num++;
				}
			}
			calUMAvgRating[userListNew.indexOf(uid)] = s / num;
		}
		System.out.println("["+(new Date())+"]_"+"calculate average rating of users is completed");

		// predicting rating of all movies rated by all users
		for (int u1 = 0; u1 < U; u1++) {
			for (int m = 0; m < M; m++) {
				float fz = 0, fm = 0;
				for (int i = 0; i < S; i++) {
					if (!allMovies[m].users.containsKey(allUsers[i]))
						continue;
					fz += ss[i][u1]
							* (allMovies[m].users.get(allUsers[i]) - calUMAvgRating[i]);
					fm += Math.abs(ss[i][u1]);
				}
				preUMRating[u1][m] = calUMAvgRating[u1] + fz / fm;
			}
		}
		System.out.println("["+(new Date())+"]_"+"preUMRating is completed");
		// predicting all rating of movies by all groups
		switch (policy) {
		case wtd:
			for (int g = 0; g < G; g++) {
				float sumOfNum = 0;
				for (int n : allGroups[g].users.values())
					sumOfNum += n;
				for (int m = 0; m < M; m++) {
					for (Entry<Integer, Integer> user : allGroups[g].users
							.entrySet()) {
						preGMRating[g][m] += user.getValue()
								/ sumOfNum
								* preUMRating[userListNew
										.indexOf(user.getKey())][m];
					}
				}
			}
			break;
		case lm:
			for (int g = 0; g < G; g++) {
				for (int m = 0; m < M; m++) {
					preGMRating[g][m] = Integer.MAX_VALUE;
					for (Entry<Integer, Integer> user : allGroups[g].users
							.entrySet()) {
						preGMRating[g][m] = Math
								.min(preGMRating[g][m], preUMRating[userListNew
										.indexOf(user.getKey())][m]);
					}
				}
			}
			break;
		case avg:
			for (int g = 0; g < G; g++) {
				for (int m = 0; m < M; m++) {
					preGMRating[g][m] = 0;
					for (Entry<Integer, Integer> user : allGroups[g].users
							.entrySet()) {
						preGMRating[g][m] += preUMRating[userListNew
								.indexOf(user.getKey())][m];
					}
					preGMRating[g][m] /= allGroups[g].users.size();
				}
			}
			break;
		}

		printResults(Model.user,policy);
		System.out.println("user-based model:");
		maeAucPcsMapRecall();
	}

	/**
	 * item-based prediction(3rd) & item-based model(4th)
	 * @param policy
	 */
	public void thirdAndFourthModel(Policy policy){

		float[] smm = new float[M+U]; // random matrix
		
		S=M;
		ss = new float[S][M];
		ssIndex = new int[S][M];
		for (int m = 0; m < M; m++) {
			System.out.print(","+m);
			for (int Urwr = 0; Urwr < (M+U); Urwr++)
				smm[Urwr] = f.nextFloat(); // random matrix
			for (int iter = 0; iter < I; iter++) {
				temp = new float[M+U];
				//temp[0...M-1] = sMGU x smm[M...M+U-1]
				for (int i = 0; i < M; i++) {
					for (int j = 0; j < U; j++) {
						temp[i] += sMGU[i][j] * smm[M+j];
					}
					temp[i] = 0.95f * temp[i] + 0.05f * (m == i ? 1 : 0);
				}
				//temp[M...M+U-1] = sUM x smm[0...M-1]
				for(int i=0;i<U;i++){
					for(int j=0;j<M;j++){
						temp[M+i] += sUM[i][j] * smm[j];
					}
					temp[M+i] *= 0.95f;
				}
				if(isConverged(smm, temp)){
					smm = temp;
					System.out.print("c");
					break;
				}
				else{
					smm = temp;
				}
			}
			for (int i = 0; i < S; i++) {
				int index = findIndexOfLargest(smm, 0, M);
				ss[i][m] = smm[index];
				ssIndex[i][m] = index;
				smm[index] = 0;
			}
		}
		System.out.println();
		System.out.println("["+(new Date())+"]_"+"ss for movie is completed");
		
		// predicting ratings of movies by users
		for(int uid:userListNew){
			float fz = 0,fm = 0;
			for(int m1=0;m1<M;m1++){
				for(int m2=0;m2<S;m2++){
					if(!allMovies[ssIndex[m2][m1]].users.containsKey(uid))
						continue;
					fz += ss[m1][m2]*allMovies[ssIndex[m2][m1]].users.get(uid);
					fm += ss[m1][m2];
				}
				preUMRating[userListNew.indexOf(uid)][m1] = fz/fm;
			}
		}
		System.out.println("["+(new Date())+"]_"+"preUMRating completed!");
	
		// predicting all rating of movies by all groups
		switch (policy) {
		case wtd:
			for (int g = 0; g < G; g++) {
				float sumOfNum = 0;
				for (int n : allGroups[g].users.values())
					sumOfNum += n;
				for (int m = 0; m < M; m++) {
					for (Entry<Integer, Integer> user : allGroups[g].users.entrySet()) {
						preGMRating[g][m] += user.getValue()
								/ sumOfNum
								* preUMRating[userListNew.indexOf(user.getKey())][m];
					}
				}
			}
			break;
		case lm:
			for (int g = 0; g < G; g++) {
				for (int m = 0; m < M; m++) {
					preGMRating[g][m] = Integer.MAX_VALUE;
					for (Entry<Integer, Integer> user : allGroups[g].users.entrySet()) {
						preGMRating[g][m] = Math.min(preGMRating[g][m], preUMRating[userListNew.indexOf(user.getKey())][m]);
					}
				}
			}
			break;
		case avg:
			for (int g = 0; g < G; g++) {
				for (int m = 0; m < M; m++) {
					preGMRating[g][m] = 0;
					for (Entry<Integer, Integer> user : allGroups[g].users.entrySet()) {
						preGMRating[g][m] += preUMRating[userListNew.indexOf(user.getKey())][m];
					}
					preGMRating[g][m] /= allGroups[g].users.size();
				}
			}
			break;
		}
		
		printResults(Model.item_pre, policy);
		System.out.println("item-based prediction:");
		maeAucPcsMapRecall();
		
		//item-based model(4th model)
		
		calGMRating(policy);
		//predicting ratings of movies by groups
		for(int g=0;g<G;g++){
			float fz = 0,fm = 0;
			for(int m1=0;m1<M;m1++){
				for(int m2=0;m2<S;m2++){
					if(calGMRating[g][ssIndex[m2][m1]] == -1)
						continue;
					fz += ss[m1][m2]*calGMRating[g][ssIndex[m2][m1]];
					fm += ss[m1][m2];
				}
				preGMRating[g][m1] = fz/fm;
			}
		}

		printResults(Model.item_mod,policy);
		System.out.println("item-based model:");
		maeAucPcsMapRecall();
	}
	
	/**
	 * 计算mae，auc，pcs,map
	 */
	public void maeAucPcsMapRecall(){
		// MAE
		float totalmae = 0;
		for (Group testGroup : groupListCheck.values()) {
			float mae = 0;
			for (Movie testMovie : movieListCheck.values()) {
				float i = preGMRating[indexOfGroup(testGroup.gid)][indexOfMovie(testMovie.mid)];
				if (test_gmRating.get(testGroup.gid).containsKey(testMovie.mid))
					i -= test_gmRating.get(testGroup.gid).get(testMovie.mid);
				mae += Math.abs(i);
			}
			mae /= test_gmRating.get(testGroup.gid).size();
			if(Float.isNaN(mae)){
				System.out.println("NaN: MAE,gid="+testGroup.gid);
//				System.exit(-1);
			}
			totalmae += mae;
		}
		totalmae /= GT;
		System.out.println("mae: " + totalmae);

		// AUC
		float result = 0;
		for (Group testGroup : groupListCheck.values()) {
			float rank = 0, pos = 0, neg = 0;
			Set<Integer> ratedMovies = new HashSet<Integer>();
			for (Movie testMovie : movieListCheck.values()) {
				for (int uid : testGroup.users.keySet()) {
					if (testMovie.users.containsKey(uid)) {
						ratedMovies.add(testMovie.mid);
						rank += rank(indexOfGroup(testGroup.gid),
								indexOfMovie(testMovie.mid));
						break;
					}
				}
			}
			pos = ratedMovies.size();
			neg = MT - pos;
			result += (rank - pos * (pos + 1) / 2) / (pos * neg);
		}
		result /= GT;
		System.out.println("AUC: " + result);

		//precision
		float totalPcs = 0;
		for (Group testGroup : groupListCheck.values()) {
			float pcs = 0;
			for (Movie testMovie : movieListCheck.values()) {
				if(!test_gmRating.get(testGroup.gid).containsKey(testMovie.mid))
					continue;
				float dif = preGMRating[indexOfGroup(testGroup.gid)][indexOfMovie(testMovie.mid)] - test_gmRating.get(testGroup.gid).get(testMovie.mid);
				float fm = Math.max(Math.abs(max-test_gmRating.get(testGroup.gid).get(testMovie.mid)),
									  Math.abs(min-test_gmRating.get(testGroup.gid).get(testMovie.mid)));
				pcs += Math.abs(dif)/fm;
			}
			totalPcs += 1-pcs/numOfMoviesRatedByGroup(testGroup.gid);
			if(Float.isNaN(totalPcs)){
				System.out.println("NaN: PCS,gid="+testGroup.gid);
//				System.exit(-1);
			}
		}
		totalPcs /= GT;
		System.out.println("precision: " + totalPcs);
		
		//MAP
		float map = 0;
		for(Group testGroup:groupListCheck.values()){
			Set<Integer> ratedMovies = new HashSet<Integer>();
			for(Movie testMovie:movieListCheck.values()){
				for(int uid:testGroup.users.keySet()){
					if(testMovie.users.containsKey(uid)){
						ratedMovies.add(testMovie.mid);
						break;
					}
				}
			}
			float count = 1,rank=0,value = 0;
			for(int ratedMovieId:ratedMovies){
				rank = MT-rank(indexOfGroup(testGroup.gid),indexOfMovie(ratedMovieId))+1;
				value += count++/rank;
			}
			value /= ratedMovies.size();
			map += value;
		}
		map /= GT;
		System.out.println("map: " + map);
		
		//precision_new & recall (k=20,40,60,...,600)
		int[] movies = new int[MT];//保存mid
		float[] ratings = new float[MT];
		int[] maxMovies;
		for(int count=10;count<=500;count+=10){
			float pcs = 0, recall = 0;
			maxMovies = new int[count];
			for(Group testGroup:groupListCheck.values()){
				int i=0;
				for(Movie testMovie:movieListCheck.values()){
					movies[i] = testMovie.mid;
					ratings[i++] = preGMRating[indexOfGroup(testGroup.gid)][indexOfMovie(testMovie.mid)];
				}
				int n=0,num=0;
				for(i=0;i<count;i++){
					int index = findIndexOfLargest(ratings, 0, ratings.length);
					maxMovies[i] = movies[index];
					ratings[index] = -1;
					for(int uid:testGroup.users.keySet()){
						if(num==0){
							for(Movie mv:movieListCheck.values()){
								if(mv.users.containsKey(uid))
									num++;
							}
						}
						if(movieListCheck.get(maxMovies[i]).users.containsKey(uid)){
							n++;
							break;
						}
					}
				}
				pcs += (float)n/(float)count;
				recall += (float)n/(float)num;				
			}
			pcs /= GT;
			recall /= GT;
			System.out.println("k="+count+": precision="+pcs+" recall="+recall);
		}
		
	}
	
	/**
	 * read from file and store to matrix
	 * @param matrix
	 * @param fileName
	 * @throws IOException
	 */
	public void fileToMatrix(float[][] matrix,String fileName) throws IOException{// 读取数据存入矩阵
		File file = new File(fileName);
		FileInputStream fIStream = new FileInputStream(file);
		InputStreamReader isr = new InputStreamReader(fIStream);
		BufferedReader br = new BufferedReader(isr);
		String text = null;
		String[] row = null;
		int r=0;
		while ((text = br.readLine()) != null) {
			System.out.println("row:" + r); 
			row = text.split(",");
			matrix[r] = new float[row.length];
			for(int i = 0;i<matrix[r].length;i++){
				matrix[r][i] = Float.parseFloat(row[i]);
			}
			r++;
		}
		br.close();
		isr.close();
		fIStream.close();
	}
	
	public static void matrixToFile(float[][] matrix,String fileName) throws Exception{
		File file = new File(fileName);
		FileOutputStream fOStream = new FileOutputStream(file);
		OutputStreamWriter osr = new OutputStreamWriter(fOStream);
		BufferedWriter bw = new BufferedWriter(osr);
		for (int i = 0; i < matrix.length; i++) {
			bw.write(String.valueOf(matrix[i][0]));
			for (int j = 1; j < matrix[i].length; j++) {
//				if (Float.isNaN(matrix[i][j])){
//					System.out.println("NaN:"+fileName.substring(fileName.lastIndexOf('/')+1,fileName.lastIndexOf('.'))+"(" + i + "," + j + ")");
//					System.exit(-1);
//				}
				bw.write("," + String.valueOf(matrix[i][j]));
			}
			bw.write("\n");
		}
		bw.close();
		osr.close();
		fOStream.close();
	}
	
	/**
	 * 找gid在allGroups中的索引
	 * 
	 * @param gid
	 * @return
	 */
	public int indexOfGroup(int gid) {
		for (int i = 0; i < allGroups.length; i++) {
			if (allGroups[i].gid == gid)
				return i;
		}
		return -1;
	}

	public int indexOfMovie(int mid) {
		for (int i = 0; i < allMovies.length; i++) {
			if (allMovies[i].mid == mid)
				return i;
		}
		return -1;
	}

	// m在g中的rating从小到大的排名(最小为1)
	private int rank(int g, int m) {
		int x = 1;
		for (int curM=0;curM<M;curM++) {
			if (movieListCheck.containsKey(allMovies[curM].mid) && preGMRating[g][curM] < preGMRating[g][m])
				x++;
		}
		return x;
	}

	public void writeOutput(String filename) throws Exception {
		
		calculatePublicModel();
		System.out.println("lm policy...");  //策略选择 
		calculateTestGMRating(Policy.lm);
		rATM_unRWR(Policy.lm);
	}

	@Override
	public void readDocs(String filename) throws Exception {
		System.out.println("Reading input...");
		try {
			Class.forName("com.mysql.jdbc.Driver"); // 加载MYSQL JDBC驱动程序

			System.out.println("Success loading Mysql Driver!");
		} catch (Exception e) {
			System.out.print("Error loading Mysql Driver!");
			e.printStackTrace();
		}

		try {
			// 连接URL为 jdbc:mysql//服务器地址/数据库名 ，后面的2个参数分别是登陆用户名和密码
			Connection connect = DriverManager.getConnection(
					"jdbc:mysql://localhost:3306/rs_train0325_group", "root", "123456");
			//"jdbc:mysql://localhost:3306/rs_train0123_group", "root", "123456");
			Connection testconnect = DriverManager.getConnection(
					"jdbc:mysql://localhost:3306/rs_check0325_group", "root", "123456");
			//"jdbc:mysql://localhost:3306/rs_new_check_group", "root", "123456");
			System.out.println("Success connect Mysql server!");

			Statement stmt = connect.createStatement();
			Statement check_stmt = testconnect.createStatement();
			int uid, mid, wid, gid;
			float rating;

			// user
			ResultSet urs = stmt.executeQuery("select * from user");
			while (urs.next()) {
				userListNew.add(urs.getInt("userid"));
			}
			ResultSet check_urs = check_stmt.executeQuery("select * from user");
			while (check_urs.next()) {
				userListCheck.add(check_urs.getInt("userid"));
			}

			// movie,movie-word
			ResultSet mrs = stmt.executeQuery("select * from movie");
			while (mrs.next()) {
				mid = mrs.getInt("movieid");
				movieListNew.put(mid, new Movie(mid)); // ??????????????
			}
			ResultSet mwrs = stmt.executeQuery("select * from movie_word");
			while (mwrs.next()) {
				mid = mwrs.getInt("movieid");
				wid = mwrs.getInt("wordid");
				movieListNew.get(mid).words.put(wid, -1);//mid->wordid->topicid
			}
			ResultSet check_mrs = check_stmt
					.executeQuery("select * from movie");
			while (check_mrs.next()) {
				mid = check_mrs.getInt("movieid");
				movieListCheck.put(mid, new Movie(mid));
			}

			// movie-user
			ResultSet murs = stmt.executeQuery("select * from user_movie");
			while (murs.next()) {
				uid = murs.getInt("userid");
				mid = murs.getInt("movieid");
				rating = murs.getInt("value")/divisor;
				movieListNew.get(mid).users.put(uid, rating);
			}
			ResultSet check_murs = check_stmt
					.executeQuery("select * from user_movie");
			while (check_murs.next()) {
				uid = check_murs.getInt("userid");
				mid = check_murs.getInt("movieid");
				rating = check_murs.getInt("value")/divisor;
				movieListCheck.get(mid).users.put(uid, rating);
			}

			// group,group-user
			ResultSet grs = stmt
					.executeQuery("select householdid from household ");
			while (grs.next()) {
				gid = grs.getInt("householdid");
				groupListNew.put(gid, new Group(gid));
			}
			ResultSet gurs = stmt.executeQuery("select * from household_user");
			while (gurs.next()) {
				gid = gurs.getInt("householdid");
				uid = gurs.getInt("userid");
				groupListNew.get(gid).users.put(uid, numOfMovies(uid));
				if (userListCheck.contains(uid)) {
					if (!groupListCheck.containsKey(gid))
						groupListCheck.put(gid, new Group(gid));
					groupListCheck.get(gid).users.put(uid, 0);
				}
			}
			for(Group testGroup:groupListCheck.values()){
				for(Entry<Integer, Integer> user:testGroup.users.entrySet()){
					for(Movie testMovie:movieListCheck.values()){
						if(testMovie.users.containsKey(user.getKey()))
							testGroup.users.put(user.getKey(), testGroup.users.get(user.getKey())+1);
					}
				}
			}

		} catch (Exception e) {
			System.out.print("get data error!");
			e.printStackTrace();
		}
		allMovies = movieListNew.values().toArray(new Movie[0]);
		allGroups = groupListNew.values().toArray(new Group[0]);
		allUsers = userListNew.toArray(new Integer[0]);

		M = allMovies.length;
		G = allGroups.length;
		U = userListNew.size();
		GT = groupListCheck.size();
		MT = movieListCheck.size();
		for (Movie movie : allMovies)
			W += movie.words.size();
		System.out.println("M:" + M + ",G:" + G + ",W:" + W + "U:" + U);
		
		preUMRating = new float[U][M];
		preGMRating = new float[G][M];// 1.13修改，把GT、MT分别改为G、M

	}

	@Override
	public void likelihood() {
		float llh = 0;
		for (Movie movie : allMovies) {
			for (Entry<Integer, Integer> word : movie.words.entrySet()) {
				float mAa = 0;
				int w = word.getKey();
				for (int z = 0; z < Z; z++) {
					mAa += ((nZW(z, w) + beta) / (nZ[z] + W * beta)) 
							* ((movie.getTopicCount(z) + alpha) / (movie.countToAllTopic + Z * alpha));
					if (mAa > 1) {
						System.out.println("nZWU: " + nZW(z, w) + ", nZU: " + nZ[z]);
						System.out.println(mAa);
					}
				}

				llh += Math.log(mAa);
			}
		}
	//System.out.println("Log-Likelihood: " + llh);
		System.out.println(llh);
	}

	/**
	 * 找a[l]到a[r-1]中最大值的index
	 * 
	 * @param a
	 * @return
	 */
	public int findIndexOfLargest(float[] a, int left, int right) {
		float largest = a[left];
		int index = left;
		for (int i = left+1; i < right; i++) {
			if (a[i] > largest) {
				largest = a[i];
				index = i;
			}
		}
		return index;
	}

	/**
	 * 查找此user打过分的所有movie
	 * 
	 * @param uid
	 * @return
	 */
	public List<Movie> findAllMoviesByUser(int uid) {
		List<Movie> movies = new ArrayList<Movie>();
		for (Movie m : movieListNew.values()) {
			if (m.isRatedByUser(uid))
				movies.add(m);
		}
		return movies;
	}

	/**
	 * 查找此user打过分的movie个数
	 * 
	 * @param uid
	 * @return
	 */
	public int numOfMovies(int uid) {
		int n = 0;
		for (Movie m : movieListNew.values()) {
			if (m.isRatedByUser(uid))
				n++;
		}
		return n;
	}

	/**
	 * 查找对此movie打过分的所有group
	 * 
	 * @param mid
	 * @return
	 */
	public List<Group> findAllGroupsByMovie(int mid) {
		List<Group> groups = new ArrayList<Group>();
		for (int uid : movieListNew.get(mid).users.keySet()) {
			for (Group g : groupListNew.values()) {
				if (g.users.containsKey(uid))
					groups.add(g);
			}
		}
		return groups;
	}

	public int numOfMoviesRatedByGroup(int gid) {
		int n = 0;
		for (Movie movie : movieListNew.values()) {
			for (int uid : groupListNew.get(gid).users.keySet()) {
				if (movie.users.containsKey(uid)) {
					n++;
					break;
				}
			}
		}
		return n;
	}
}
